package umich.max.geolocation.featextract.icf;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map.Entry;

import umich.max.geolocation.util.BlogUtils;

import max.nlp.dal.blog.blogger.BlogAuthorProfile;
import max.nlp.dal.blog.blogger.BloggerBlog;
import max.nlp.dal.blog.blogger.BloggerDB;
import max.nlp.dal.blog.blogger.BloggerPost;
import max.nlp.dal.blog.blogger.ParsedLocation;
import max.nlp.util.MapSorter;
import max.nlp.wrappers.ml.weka.FeatureSaver;
import max.nlp.wrappers.stanford.StanfordNLP;

import com.mongodb.DBCursor;

import edu.stanford.nlp.util.CoreMap;

public class ICFComputer {

	private int numStates;
	private static final StanfordNLP tokenizer = StanfordNLP.getInstance();
	private HashMap<String, ArrayList<String>> words = new HashMap<String, ArrayList<String>>();
	private HashMap<String, Double> icf = new HashMap<String, Double>();

	public HashMap<String, Double> getIcf() {
		return icf;
	}

	public void setIcf(HashMap<String, Double> icf) {
		this.icf = icf;
	}

	public static void main(String[] args) {
		ICFComputer c = new ICFComputer();
		c.generateInitialICF();
	}

	public void generateInitialICF() {
		numStates = BlogUtils.getAllStates().size();
		BloggerDB db = BloggerDB.getInstance();
		DBCursor itr = db.getProfileIterator();
		int count = 0;
		while (itr.hasNext()) {

			if (count % 100 == 0)
				FeatureSaver.saveVector("icfs3-states-" + count, getIcf());
			BlogAuthorProfile profile = new BlogAuthorProfile(itr.next());
			ParsedLocation loc = profile.getParsedLocation();
			if (loc != null) {
				String state = loc.getState();
				if (state != null) {
					for (String blogURL : profile.getBlogs()) {
						BloggerBlog blog = db.findBlogbyURL(blogURL);
						if (blog != null) {
							List<BloggerPost> posts = blog.getPosts();
							if (posts != null && !posts.isEmpty()) {
								for (BloggerPost p : posts) {
									processPost(p, state);
								}
							}
						}
					}
				}
			}
		}
		convertToICF();
		FeatureSaver.saveVector("icfs3-states", getIcf());

	}

	@SuppressWarnings("serial")
	public void processPost(BloggerPost p, final String state) {
		String text = p.getCleanContent();
		List<CoreMap> annotate = tokenizer.annotate(text);

		for (CoreMap sentence : annotate) {
			LinkedHashMap<String, String> tokenizedSentence = tokenizer
					.extractTaggedTokensFromLabeledSentence("tokenization",
							sentence);
			for (Entry<String, String> e : tokenizedSentence.entrySet()) {
				String word = e.getKey();
				List<String> statesWithWord = words.get(word);
				if (statesWithWord == null) {
					words.put(word, new ArrayList<String>() {
						{
							add(state);
						}
					});
				} else {
					statesWithWord.add(state);
				}
			}
		}

	}

	public void convertToICF() {
		double n = (double) numStates;
		for (Entry<String, ArrayList<String>> e : words.entrySet()) {
			String term = e.getKey();
			double cf = (double) e.getValue().size();
			icf.put(term, n / cf);
		}
	}

	public void printICF() {
		MapSorter<String, Double> m = new MapSorter<String, Double>();
		List<Entry<String, Double>> sorted = m.sortMap(icf);

		for (Entry<String, Double> e : sorted) {
			System.out.println(e.getKey() + " - " + e.getValue());
		}
	}
}
